{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 过滤数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "from pathlib import Path\n",
    "import os \n",
    "from PIL import Image\n",
    "import shutil\n",
    "from torch import Tensor\n",
    "from glob import glob\n",
    "from tqdm import tqdm \n",
    "from joblib import Parallel, delayed\n",
    "from typing import Union\n",
    "from torchvision.io import ImageReadMode, read_image\n",
    "from transformers import ChineseCLIPProcessor, ChineseCLIPModel,ViTImageProcessor\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bigdata/image_data\\\\test-0.jpg', 'bigdata/image_data\\\\test-1.jpg']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_image_path_list = glob(\"bigdata/image_data/*\")\n",
    "all_image_path_list[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 224, 224])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "VIT_MODEL_NAME_OR_PATH = \"google/vit-base-patch16-224\"\n",
    "processor = ViTImageProcessor.from_pretrained(VIT_MODEL_NAME_OR_PATH)\n",
    "\n",
    "def process_image_2_pixel_value(x:str) -> Tensor:\n",
    "    image = Image.open(x)\n",
    "    res = processor(images=image, return_tensors='pt')['pixel_values'].squeeze(0)\n",
    "    return res \n",
    "\n",
    "\n",
    "process_image_2_pixel_value(x = \"bigdata/image_data/test-9282.jpg\").shape "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 20 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   1 tasks      | elapsed:   10.0s\n",
      "[Parallel(n_jobs=-1)]: Done  10 tasks      | elapsed:   10.6s\n",
      "[Parallel(n_jobs=-1)]: Done  21 tasks      | elapsed:   10.8s\n",
      "[Parallel(n_jobs=-1)]: Done  32 tasks      | elapsed:   11.1s\n",
      "[Parallel(n_jobs=-1)]: Done  45 tasks      | elapsed:   11.5s\n",
      "[Parallel(n_jobs=-1)]: Done  58 tasks      | elapsed:   11.8s\n",
      "[Parallel(n_jobs=-1)]: Done  73 tasks      | elapsed:   12.1s\n",
      "[Parallel(n_jobs=-1)]: Done  88 tasks      | elapsed:   12.3s\n",
      "[Parallel(n_jobs=-1)]: Done 105 tasks      | elapsed:   12.4s\n",
      "[Parallel(n_jobs=-1)]: Done 122 tasks      | elapsed:   12.6s\n",
      "[Parallel(n_jobs=-1)]: Done 141 tasks      | elapsed:   12.8s\n",
      "[Parallel(n_jobs=-1)]: Done 160 tasks      | elapsed:   13.0s\n",
      "[Parallel(n_jobs=-1)]: Done 181 tasks      | elapsed:   13.2s\n",
      "[Parallel(n_jobs=-1)]: Done 202 tasks      | elapsed:   13.4s\n",
      "[Parallel(n_jobs=-1)]: Done 225 tasks      | elapsed:   13.6s\n",
      "[Parallel(n_jobs=-1)]: Done 248 tasks      | elapsed:   13.7s\n",
      "[Parallel(n_jobs=-1)]: Done 273 tasks      | elapsed:   13.8s\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.1973s.) Setting batch_size=2.\n",
      "[Parallel(n_jobs=-1)]: Done 298 tasks      | elapsed:   13.9s\n",
      "[Parallel(n_jobs=-1)]: Done 325 tasks      | elapsed:   14.1s\n",
      "[Parallel(n_jobs=-1)]: Batch computation too fast (0.1684s.) Setting batch_size=4.\n",
      "[Parallel(n_jobs=-1)]: Done 364 tasks      | elapsed:   14.3s\n",
      "[Parallel(n_jobs=-1)]: Done 426 tasks      | elapsed:   14.6s\n",
      "[Parallel(n_jobs=-1)]: Done 540 tasks      | elapsed:   14.9s\n",
      "[Parallel(n_jobs=-1)]: Done 664 tasks      | elapsed:   15.3s\n",
      "[Parallel(n_jobs=-1)]: Done 788 tasks      | elapsed:   15.6s\n",
      "[Parallel(n_jobs=-1)]: Done 920 tasks      | elapsed:   16.0s\n",
      "[Parallel(n_jobs=-1)]: Done 1052 tasks      | elapsed:   16.4s\n",
      "[Parallel(n_jobs=-1)]: Done 1192 tasks      | elapsed:   16.7s\n",
      "[Parallel(n_jobs=-1)]: Done 1332 tasks      | elapsed:   17.0s\n",
      "[Parallel(n_jobs=-1)]: Done 1480 tasks      | elapsed:   17.3s\n",
      "[Parallel(n_jobs=-1)]: Done 1628 tasks      | elapsed:   17.6s\n",
      "[Parallel(n_jobs=-1)]: Done 1784 tasks      | elapsed:   18.0s\n",
      "[Parallel(n_jobs=-1)]: Done 1940 tasks      | elapsed:   18.4s\n",
      "[Parallel(n_jobs=-1)]: Done 2104 tasks      | elapsed:   18.7s\n",
      "[Parallel(n_jobs=-1)]: Done 2268 tasks      | elapsed:   19.1s\n",
      "[Parallel(n_jobs=-1)]: Done 2440 tasks      | elapsed:   19.5s\n",
      "[Parallel(n_jobs=-1)]: Done 2612 tasks      | elapsed:   19.9s\n",
      "[Parallel(n_jobs=-1)]: Done 2792 tasks      | elapsed:   20.5s\n",
      "[Parallel(n_jobs=-1)]: Done 2972 tasks      | elapsed:   20.9s\n",
      "[Parallel(n_jobs=-1)]: Done 3160 tasks      | elapsed:   21.3s\n",
      "[Parallel(n_jobs=-1)]: Done 3348 tasks      | elapsed:   21.8s\n",
      "[Parallel(n_jobs=-1)]: Done 3544 tasks      | elapsed:   22.3s\n",
      "[Parallel(n_jobs=-1)]: Done 3740 tasks      | elapsed:   22.8s\n",
      "[Parallel(n_jobs=-1)]: Done 3944 tasks      | elapsed:   23.3s\n",
      "[Parallel(n_jobs=-1)]: Done 4148 tasks      | elapsed:   23.8s\n",
      "[Parallel(n_jobs=-1)]: Done 4360 tasks      | elapsed:   24.3s\n",
      "[Parallel(n_jobs=-1)]: Done 4572 tasks      | elapsed:   25.0s\n",
      "[Parallel(n_jobs=-1)]: Done 4792 tasks      | elapsed:   25.6s\n",
      "[Parallel(n_jobs=-1)]: Done 5012 tasks      | elapsed:   26.3s\n",
      "[Parallel(n_jobs=-1)]: Done 5240 tasks      | elapsed:   27.0s\n",
      "[Parallel(n_jobs=-1)]: Done 5468 tasks      | elapsed:   27.7s\n",
      "[Parallel(n_jobs=-1)]: Done 5704 tasks      | elapsed:   28.3s\n",
      "[Parallel(n_jobs=-1)]: Done 5940 tasks      | elapsed:   28.9s\n",
      "[Parallel(n_jobs=-1)]: Done 6184 tasks      | elapsed:   29.5s\n",
      "[Parallel(n_jobs=-1)]: Done 6428 tasks      | elapsed:   30.2s\n",
      "[Parallel(n_jobs=-1)]: Done 6680 tasks      | elapsed:   31.1s\n",
      "[Parallel(n_jobs=-1)]: Done 6932 tasks      | elapsed:   31.8s\n",
      "[Parallel(n_jobs=-1)]: Done 7192 tasks      | elapsed:   32.6s\n",
      "[Parallel(n_jobs=-1)]: Done 7452 tasks      | elapsed:   33.4s\n",
      "[Parallel(n_jobs=-1)]: Done 7720 tasks      | elapsed:   34.3s\n",
      "[Parallel(n_jobs=-1)]: Done 7988 tasks      | elapsed:   35.1s\n",
      "[Parallel(n_jobs=-1)]: Done 8264 tasks      | elapsed:   36.0s\n",
      "[Parallel(n_jobs=-1)]: Done 8540 tasks      | elapsed:   36.8s\n",
      "[Parallel(n_jobs=-1)]: Done 8824 tasks      | elapsed:   37.6s\n",
      "[Parallel(n_jobs=-1)]: Done 9108 tasks      | elapsed:   38.4s\n",
      "[Parallel(n_jobs=-1)]: Done 9400 tasks      | elapsed:   39.2s\n",
      "[Parallel(n_jobs=-1)]: Done 9692 tasks      | elapsed:   39.9s\n",
      "[Parallel(n_jobs=-1)]: Done 9992 tasks      | elapsed:   40.7s\n",
      "[Parallel(n_jobs=-1)]: Done 10292 tasks      | elapsed:   41.5s\n",
      "[Parallel(n_jobs=-1)]: Done 10600 tasks      | elapsed:   42.2s\n",
      "[Parallel(n_jobs=-1)]: Done 10908 tasks      | elapsed:   43.0s\n",
      "[Parallel(n_jobs=-1)]: Done 11224 tasks      | elapsed:   44.0s\n",
      "[Parallel(n_jobs=-1)]: Done 11540 tasks      | elapsed:   44.9s\n",
      "[Parallel(n_jobs=-1)]: Done 11864 tasks      | elapsed:   45.8s\n",
      "[Parallel(n_jobs=-1)]: Done 12188 tasks      | elapsed:   46.7s\n",
      "[Parallel(n_jobs=-1)]: Done 12520 tasks      | elapsed:   47.7s\n",
      "[Parallel(n_jobs=-1)]: Done 12852 tasks      | elapsed:   48.6s\n",
      "[Parallel(n_jobs=-1)]: Done 13192 tasks      | elapsed:   49.6s\n",
      "[Parallel(n_jobs=-1)]: Done 13532 tasks      | elapsed:   50.4s\n",
      "[Parallel(n_jobs=-1)]: Done 13880 tasks      | elapsed:   51.3s\n",
      "[Parallel(n_jobs=-1)]: Done 14228 tasks      | elapsed:   52.3s\n",
      "[Parallel(n_jobs=-1)]: Done 14584 tasks      | elapsed:   53.2s\n",
      "[Parallel(n_jobs=-1)]: Done 14940 tasks      | elapsed:   54.2s\n",
      "[Parallel(n_jobs=-1)]: Done 15304 tasks      | elapsed:   55.2s\n",
      "[Parallel(n_jobs=-1)]: Done 15668 tasks      | elapsed:   56.1s\n",
      "[Parallel(n_jobs=-1)]: Done 16040 tasks      | elapsed:   56.9s\n",
      "[Parallel(n_jobs=-1)]: Done 16412 tasks      | elapsed:   57.7s\n",
      "[Parallel(n_jobs=-1)]: Done 16792 tasks      | elapsed:   58.7s\n",
      "[Parallel(n_jobs=-1)]: Done 17172 tasks      | elapsed:   59.6s\n",
      "[Parallel(n_jobs=-1)]: Done 17560 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 17948 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 18344 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=-1)]: Done 18740 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 19144 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 19548 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 19960 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 20372 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 20792 tasks      | elapsed:  1.1min\n",
      "[Parallel(n_jobs=-1)]: Done 21212 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 21640 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 22068 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 22504 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 22940 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 23384 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 23828 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 24280 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 24732 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 25192 tasks      | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 25652 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 26120 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 26588 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 27064 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 27540 tasks      | elapsed:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done 28024 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 28508 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 29000 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 29492 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 29992 tasks      | elapsed:  1.5min\n",
      "[Parallel(n_jobs=-1)]: Done 30492 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 31000 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 31508 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 32024 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 32540 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 33064 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 33588 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 34120 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 34652 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 35192 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 35732 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 36280 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 36828 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 37384 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 37940 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 38504 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 39068 tasks      | elapsed:  1.9min\n",
      "[Parallel(n_jobs=-1)]: Done 39640 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 40212 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 40792 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 41372 tasks      | elapsed:  2.0min\n",
      "[Parallel(n_jobs=-1)]: Done 41960 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 42548 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 43144 tasks      | elapsed:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done 43740 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 44344 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 44948 tasks      | elapsed:  2.2min\n",
      "[Parallel(n_jobs=-1)]: Done 45560 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 46172 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 46792 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 47412 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 48040 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 48668 tasks      | elapsed:  2.4min\n",
      "[Parallel(n_jobs=-1)]: Done 49304 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 49940 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 50584 tasks      | elapsed:  2.5min\n",
      "[Parallel(n_jobs=-1)]: Done 51228 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 51880 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 52532 tasks      | elapsed:  2.6min\n",
      "[Parallel(n_jobs=-1)]: Done 53192 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 53852 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 54520 tasks      | elapsed:  2.7min\n",
      "[Parallel(n_jobs=-1)]: Done 55188 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 55864 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 56540 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 57224 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 57908 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 58600 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 59292 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 59992 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 60692 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 61400 tasks      | elapsed:  3.0min\n",
      "[Parallel(n_jobs=-1)]: Done 62108 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 62824 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 63540 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 64264 tasks      | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 64988 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 65720 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 66452 tasks      | elapsed:  3.2min\n",
      "[Parallel(n_jobs=-1)]: Done 67192 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 67932 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 68680 tasks      | elapsed:  3.3min\n",
      "[Parallel(n_jobs=-1)]: Done 69428 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=-1)]: Done 70184 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=-1)]: Done 70940 tasks      | elapsed:  3.4min\n",
      "[Parallel(n_jobs=-1)]: Done 71704 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 72468 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 73240 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 74012 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=-1)]: Done 74792 tasks      | elapsed:  3.6min\n",
      "[Parallel(n_jobs=-1)]: Done 75572 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 76360 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 77148 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 77944 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 78740 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 79544 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 80348 tasks      | elapsed:  3.8min\n",
      "[Parallel(n_jobs=-1)]: Done 81160 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 81972 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 82792 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 83612 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 84440 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 85268 tasks      | elapsed:  4.0min\n",
      "[Parallel(n_jobs=-1)]: Done 86104 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=-1)]: Done 86940 tasks      | elapsed:  4.1min\n",
      "[Parallel(n_jobs=-1)]: Done 87784 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 88628 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 89480 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 90332 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 91192 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 92052 tasks      | elapsed:  4.3min\n",
      "[Parallel(n_jobs=-1)]: Done 92920 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=-1)]: Done 93788 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=-1)]: Done 94664 tasks      | elapsed:  4.4min\n",
      "[Parallel(n_jobs=-1)]: Done 95540 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=-1)]: Done 96424 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=-1)]: Done 97308 tasks      | elapsed:  4.5min\n",
      "[Parallel(n_jobs=-1)]: Done 98200 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 99092 tasks      | elapsed:  4.6min\n",
      "[Parallel(n_jobs=-1)]: Done 99992 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=-1)]: Done 100892 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=-1)]: Done 101800 tasks      | elapsed:  4.7min\n",
      "[Parallel(n_jobs=-1)]: Done 102708 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 103624 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 104540 tasks      | elapsed:  4.8min\n",
      "[Parallel(n_jobs=-1)]: Done 105464 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 106388 tasks      | elapsed:  4.9min\n",
      "[Parallel(n_jobs=-1)]: Done 107320 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=-1)]: Done 108252 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=-1)]: Done 109192 tasks      | elapsed:  5.0min\n",
      "[Parallel(n_jobs=-1)]: Done 110132 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 111080 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 112028 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done 112984 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done 113940 tasks      | elapsed:  5.2min\n",
      "[Parallel(n_jobs=-1)]: Done 114904 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=-1)]: Done 115868 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=-1)]: Done 116840 tasks      | elapsed:  5.3min\n",
      "[Parallel(n_jobs=-1)]: Done 117812 tasks      | elapsed:  5.4min\n",
      "[Parallel(n_jobs=-1)]: Done 118792 tasks      | elapsed:  5.4min\n",
      "[Parallel(n_jobs=-1)]: Done 119772 tasks      | elapsed:  5.5min\n",
      "[Parallel(n_jobs=-1)]: Done 120760 tasks      | elapsed:  5.5min\n",
      "[Parallel(n_jobs=-1)]: Done 121748 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 122744 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 123740 tasks      | elapsed:  5.6min\n",
      "[Parallel(n_jobs=-1)]: Done 124744 tasks      | elapsed:  5.7min\n",
      "[Parallel(n_jobs=-1)]: Done 125748 tasks      | elapsed:  5.7min\n",
      "[Parallel(n_jobs=-1)]: Done 126760 tasks      | elapsed:  5.8min\n",
      "[Parallel(n_jobs=-1)]: Done 127772 tasks      | elapsed:  5.8min\n",
      "[Parallel(n_jobs=-1)]: Done 128792 tasks      | elapsed:  5.9min\n",
      "[Parallel(n_jobs=-1)]: Done 129812 tasks      | elapsed:  5.9min\n",
      "[Parallel(n_jobs=-1)]: Done 130840 tasks      | elapsed:  5.9min\n",
      "[Parallel(n_jobs=-1)]: Done 131868 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 132904 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 133940 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 134984 tasks      | elapsed:  6.1min\n",
      "[Parallel(n_jobs=-1)]: Done 136028 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=-1)]: Done 137080 tasks      | elapsed:  6.2min\n",
      "[Parallel(n_jobs=-1)]: Done 138132 tasks      | elapsed:  6.3min\n",
      "[Parallel(n_jobs=-1)]: Done 139192 tasks      | elapsed:  6.3min\n",
      "[Parallel(n_jobs=-1)]: Done 140252 tasks      | elapsed:  6.4min\n",
      "[Parallel(n_jobs=-1)]: Done 141320 tasks      | elapsed:  6.4min\n",
      "[Parallel(n_jobs=-1)]: Done 142388 tasks      | elapsed:  6.5min\n",
      "[Parallel(n_jobs=-1)]: Done 143464 tasks      | elapsed:  6.5min\n",
      "[Parallel(n_jobs=-1)]: Done 144540 tasks      | elapsed:  6.6min\n",
      "[Parallel(n_jobs=-1)]: Done 145624 tasks      | elapsed:  6.6min\n",
      "[Parallel(n_jobs=-1)]: Done 146708 tasks      | elapsed:  6.6min\n",
      "[Parallel(n_jobs=-1)]: Done 147800 tasks      | elapsed:  6.7min\n",
      "[Parallel(n_jobs=-1)]: Done 148892 tasks      | elapsed:  6.7min\n",
      "[Parallel(n_jobs=-1)]: Done 149992 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 151092 tasks      | elapsed:  6.8min\n",
      "[Parallel(n_jobs=-1)]: Done 152200 tasks      | elapsed:  6.9min\n",
      "[Parallel(n_jobs=-1)]: Done 153308 tasks      | elapsed:  6.9min\n",
      "[Parallel(n_jobs=-1)]: Done 154424 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done 155540 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done 156664 tasks      | elapsed:  7.0min\n",
      "[Parallel(n_jobs=-1)]: Done 157788 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=-1)]: Done 158920 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=-1)]: Done 160052 tasks      | elapsed:  7.2min\n",
      "[Parallel(n_jobs=-1)]: Done 161192 tasks      | elapsed:  7.2min\n",
      "[Parallel(n_jobs=-1)]: Done 162332 tasks      | elapsed:  7.3min\n",
      "[Parallel(n_jobs=-1)]: Done 163480 tasks      | elapsed:  7.3min\n",
      "[Parallel(n_jobs=-1)]: Done 164628 tasks      | elapsed:  7.4min\n",
      "[Parallel(n_jobs=-1)]: Done 165784 tasks      | elapsed:  7.4min\n",
      "[Parallel(n_jobs=-1)]: Done 166940 tasks      | elapsed:  7.5min\n",
      "[Parallel(n_jobs=-1)]: Done 168104 tasks      | elapsed:  7.5min\n",
      "[Parallel(n_jobs=-1)]: Done 169268 tasks      | elapsed:  7.6min\n",
      "[Parallel(n_jobs=-1)]: Done 170440 tasks      | elapsed:  7.6min\n",
      "[Parallel(n_jobs=-1)]: Done 171612 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=-1)]: Done 172792 tasks      | elapsed:  7.7min\n",
      "[Parallel(n_jobs=-1)]: Done 173972 tasks      | elapsed:  7.8min\n",
      "[Parallel(n_jobs=-1)]: Done 175160 tasks      | elapsed:  7.8min\n",
      "[Parallel(n_jobs=-1)]: Done 176348 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=-1)]: Done 177544 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=-1)]: Done 178740 tasks      | elapsed:  8.0min\n",
      "[Parallel(n_jobs=-1)]: Done 179944 tasks      | elapsed:  8.0min\n",
      "[Parallel(n_jobs=-1)]: Done 181148 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=-1)]: Done 182360 tasks      | elapsed:  8.1min\n",
      "[Parallel(n_jobs=-1)]: Done 183572 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=-1)]: Done 184792 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=-1)]: Done 186012 tasks      | elapsed:  8.3min\n",
      "[Parallel(n_jobs=-1)]: Done 187240 tasks      | elapsed:  8.3min\n",
      "[Parallel(n_jobs=-1)]: Done 188468 tasks      | elapsed:  8.4min\n",
      "[Parallel(n_jobs=-1)]: Done 189704 tasks      | elapsed:  8.5min\n",
      "[Parallel(n_jobs=-1)]: Done 190940 tasks      | elapsed:  8.5min\n",
      "[Parallel(n_jobs=-1)]: Done 192184 tasks      | elapsed:  8.6min\n",
      "[Parallel(n_jobs=-1)]: Done 193428 tasks      | elapsed:  8.7min\n",
      "[Parallel(n_jobs=-1)]: Done 194680 tasks      | elapsed:  8.8min\n",
      "[Parallel(n_jobs=-1)]: Done 195932 tasks      | elapsed:  8.8min\n",
      "[Parallel(n_jobs=-1)]: Done 197192 tasks      | elapsed:  8.9min\n",
      "[Parallel(n_jobs=-1)]: Done 198452 tasks      | elapsed:  9.0min\n",
      "[Parallel(n_jobs=-1)]: Done 199720 tasks      | elapsed:  9.0min\n",
      "[Parallel(n_jobs=-1)]: Done 200988 tasks      | elapsed:  9.1min\n",
      "[Parallel(n_jobs=-1)]: Done 202264 tasks      | elapsed:  9.2min\n",
      "[Parallel(n_jobs=-1)]: Done 203540 tasks      | elapsed:  9.2min\n",
      "[Parallel(n_jobs=-1)]: Done 204824 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=-1)]: Done 206108 tasks      | elapsed:  9.3min\n",
      "[Parallel(n_jobs=-1)]: Done 207400 tasks      | elapsed:  9.4min\n",
      "[Parallel(n_jobs=-1)]: Done 208692 tasks      | elapsed:  9.4min\n",
      "[Parallel(n_jobs=-1)]: Done 209992 tasks      | elapsed:  9.5min\n",
      "[Parallel(n_jobs=-1)]: Done 211292 tasks      | elapsed:  9.5min\n",
      "[Parallel(n_jobs=-1)]: Done 212600 tasks      | elapsed:  9.6min\n",
      "[Parallel(n_jobs=-1)]: Done 213908 tasks      | elapsed:  9.6min\n",
      "[Parallel(n_jobs=-1)]: Done 215224 tasks      | elapsed:  9.7min\n",
      "[Parallel(n_jobs=-1)]: Done 216540 tasks      | elapsed:  9.8min\n",
      "[Parallel(n_jobs=-1)]: Done 217864 tasks      | elapsed:  9.8min\n",
      "[Parallel(n_jobs=-1)]: Done 219188 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 220520 tasks      | elapsed:  9.9min\n",
      "[Parallel(n_jobs=-1)]: Done 221852 tasks      | elapsed: 10.0min\n",
      "[Parallel(n_jobs=-1)]: Done 223192 tasks      | elapsed: 10.0min\n",
      "[Parallel(n_jobs=-1)]: Done 224532 tasks      | elapsed: 10.1min\n",
      "[Parallel(n_jobs=-1)]: Done 225880 tasks      | elapsed: 10.1min\n",
      "[Parallel(n_jobs=-1)]: Done 227228 tasks      | elapsed: 10.2min\n",
      "[Parallel(n_jobs=-1)]: Done 228584 tasks      | elapsed: 10.3min\n",
      "[Parallel(n_jobs=-1)]: Done 229940 tasks      | elapsed: 10.3min\n",
      "[Parallel(n_jobs=-1)]: Done 231304 tasks      | elapsed: 10.4min\n",
      "[Parallel(n_jobs=-1)]: Done 232668 tasks      | elapsed: 10.4min\n",
      "[Parallel(n_jobs=-1)]: Done 234040 tasks      | elapsed: 10.5min\n",
      "[Parallel(n_jobs=-1)]: Done 235412 tasks      | elapsed: 10.5min\n",
      "[Parallel(n_jobs=-1)]: Done 236792 tasks      | elapsed: 10.6min\n",
      "[Parallel(n_jobs=-1)]: Done 238172 tasks      | elapsed: 10.7min\n",
      "[Parallel(n_jobs=-1)]: Done 239560 tasks      | elapsed: 10.7min\n",
      "[Parallel(n_jobs=-1)]: Done 240948 tasks      | elapsed: 10.8min\n",
      "[Parallel(n_jobs=-1)]: Done 242344 tasks      | elapsed: 10.8min\n",
      "[Parallel(n_jobs=-1)]: Done 243740 tasks      | elapsed: 10.9min\n",
      "[Parallel(n_jobs=-1)]: Done 245144 tasks      | elapsed: 10.9min\n",
      "[Parallel(n_jobs=-1)]: Done 246548 tasks      | elapsed: 11.0min\n",
      "[Parallel(n_jobs=-1)]: Done 247960 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=-1)]: Done 249372 tasks      | elapsed: 11.1min\n",
      "[Parallel(n_jobs=-1)]: Done 250792 tasks      | elapsed: 11.2min\n",
      "[Parallel(n_jobs=-1)]: Done 252212 tasks      | elapsed: 11.2min\n",
      "[Parallel(n_jobs=-1)]: Done 253640 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=-1)]: Done 255068 tasks      | elapsed: 11.3min\n",
      "[Parallel(n_jobs=-1)]: Done 256504 tasks      | elapsed: 11.4min\n",
      "[Parallel(n_jobs=-1)]: Done 257940 tasks      | elapsed: 11.5min\n",
      "[Parallel(n_jobs=-1)]: Done 259384 tasks      | elapsed: 11.5min\n",
      "[Parallel(n_jobs=-1)]: Done 260828 tasks      | elapsed: 11.6min\n",
      "[Parallel(n_jobs=-1)]: Done 262280 tasks      | elapsed: 11.7min\n",
      "[Parallel(n_jobs=-1)]: Done 263732 tasks      | elapsed: 11.7min\n",
      "[Parallel(n_jobs=-1)]: Done 265192 tasks      | elapsed: 11.8min\n",
      "[Parallel(n_jobs=-1)]: Done 266652 tasks      | elapsed: 11.9min\n",
      "[Parallel(n_jobs=-1)]: Done 268120 tasks      | elapsed: 11.9min\n",
      "[Parallel(n_jobs=-1)]: Done 269588 tasks      | elapsed: 12.0min\n",
      "[Parallel(n_jobs=-1)]: Done 271064 tasks      | elapsed: 12.0min\n",
      "[Parallel(n_jobs=-1)]: Done 272540 tasks      | elapsed: 12.1min\n",
      "[Parallel(n_jobs=-1)]: Done 274024 tasks      | elapsed: 12.2min\n",
      "[Parallel(n_jobs=-1)]: Done 275508 tasks      | elapsed: 12.2min\n",
      "[Parallel(n_jobs=-1)]: Done 277000 tasks      | elapsed: 12.3min\n",
      "[Parallel(n_jobs=-1)]: Done 278492 tasks      | elapsed: 12.4min\n",
      "[Parallel(n_jobs=-1)]: Done 279992 tasks      | elapsed: 12.4min\n",
      "[Parallel(n_jobs=-1)]: Done 281492 tasks      | elapsed: 12.5min\n",
      "[Parallel(n_jobs=-1)]: Done 283000 tasks      | elapsed: 12.6min\n",
      "[Parallel(n_jobs=-1)]: Done 284508 tasks      | elapsed: 12.6min\n",
      "[Parallel(n_jobs=-1)]: Done 286024 tasks      | elapsed: 12.7min\n",
      "[Parallel(n_jobs=-1)]: Done 287540 tasks      | elapsed: 12.7min\n",
      "[Parallel(n_jobs=-1)]: Done 289064 tasks      | elapsed: 12.8min\n",
      "[Parallel(n_jobs=-1)]: Done 290588 tasks      | elapsed: 12.9min\n",
      "[Parallel(n_jobs=-1)]: Done 292120 tasks      | elapsed: 12.9min\n",
      "[Parallel(n_jobs=-1)]: Done 293652 tasks      | elapsed: 13.0min\n",
      "[Parallel(n_jobs=-1)]: Done 295192 tasks      | elapsed: 13.1min\n",
      "[Parallel(n_jobs=-1)]: Done 296732 tasks      | elapsed: 13.1min\n",
      "[Parallel(n_jobs=-1)]: Done 298280 tasks      | elapsed: 13.2min\n",
      "[Parallel(n_jobs=-1)]: Done 299828 tasks      | elapsed: 13.3min\n",
      "[Parallel(n_jobs=-1)]: Done 301384 tasks      | elapsed: 13.4min\n",
      "[Parallel(n_jobs=-1)]: Done 302940 tasks      | elapsed: 13.4min\n",
      "[Parallel(n_jobs=-1)]: Done 304504 tasks      | elapsed: 13.5min\n",
      "[Parallel(n_jobs=-1)]: Done 306068 tasks      | elapsed: 13.6min\n",
      "[Parallel(n_jobs=-1)]: Done 307640 tasks      | elapsed: 13.6min\n",
      "[Parallel(n_jobs=-1)]: Done 309212 tasks      | elapsed: 13.7min\n",
      "[Parallel(n_jobs=-1)]: Done 310792 tasks      | elapsed: 13.7min\n",
      "[Parallel(n_jobs=-1)]: Done 312372 tasks      | elapsed: 13.8min\n",
      "[Parallel(n_jobs=-1)]: Done 313960 tasks      | elapsed: 13.9min\n",
      "[Parallel(n_jobs=-1)]: Done 315548 tasks      | elapsed: 13.9min\n",
      "[Parallel(n_jobs=-1)]: Done 317144 tasks      | elapsed: 14.0min\n",
      "[Parallel(n_jobs=-1)]: Done 318740 tasks      | elapsed: 14.1min\n",
      "[Parallel(n_jobs=-1)]: Done 320344 tasks      | elapsed: 14.1min\n",
      "[Parallel(n_jobs=-1)]: Done 321948 tasks      | elapsed: 14.2min\n",
      "[Parallel(n_jobs=-1)]: Done 323560 tasks      | elapsed: 14.3min\n",
      "[Parallel(n_jobs=-1)]: Done 325172 tasks      | elapsed: 14.3min\n",
      "[Parallel(n_jobs=-1)]: Done 326792 tasks      | elapsed: 14.4min\n",
      "[Parallel(n_jobs=-1)]: Done 328412 tasks      | elapsed: 14.5min\n",
      "[Parallel(n_jobs=-1)]: Done 330040 tasks      | elapsed: 14.5min\n",
      "[Parallel(n_jobs=-1)]: Done 331668 tasks      | elapsed: 14.6min\n",
      "[Parallel(n_jobs=-1)]: Done 333304 tasks      | elapsed: 14.7min\n",
      "[Parallel(n_jobs=-1)]: Done 334940 tasks      | elapsed: 14.7min\n",
      "[Parallel(n_jobs=-1)]: Done 336584 tasks      | elapsed: 14.8min\n",
      "[Parallel(n_jobs=-1)]: Done 338228 tasks      | elapsed: 14.9min\n",
      "[Parallel(n_jobs=-1)]: Done 339880 tasks      | elapsed: 14.9min\n",
      "[Parallel(n_jobs=-1)]: Done 341532 tasks      | elapsed: 15.0min\n",
      "[Parallel(n_jobs=-1)]: Done 343192 tasks      | elapsed: 15.1min\n",
      "[Parallel(n_jobs=-1)]: Done 344852 tasks      | elapsed: 15.1min\n",
      "[Parallel(n_jobs=-1)]: Done 346520 tasks      | elapsed: 15.2min\n",
      "[Parallel(n_jobs=-1)]: Done 348188 tasks      | elapsed: 15.3min\n",
      "[Parallel(n_jobs=-1)]: Done 349864 tasks      | elapsed: 15.3min\n",
      "[Parallel(n_jobs=-1)]: Done 351540 tasks      | elapsed: 15.4min\n",
      "[Parallel(n_jobs=-1)]: Done 353224 tasks      | elapsed: 15.5min\n",
      "[Parallel(n_jobs=-1)]: Done 354908 tasks      | elapsed: 15.5min\n",
      "[Parallel(n_jobs=-1)]: Done 356600 tasks      | elapsed: 15.6min\n",
      "[Parallel(n_jobs=-1)]: Done 358292 tasks      | elapsed: 15.7min\n",
      "[Parallel(n_jobs=-1)]: Done 359992 tasks      | elapsed: 15.7min\n",
      "[Parallel(n_jobs=-1)]: Done 361692 tasks      | elapsed: 15.8min\n",
      "[Parallel(n_jobs=-1)]: Done 363400 tasks      | elapsed: 15.9min\n",
      "[Parallel(n_jobs=-1)]: Done 365108 tasks      | elapsed: 15.9min\n",
      "[Parallel(n_jobs=-1)]: Done 366824 tasks      | elapsed: 16.0min\n",
      "[Parallel(n_jobs=-1)]: Done 368540 tasks      | elapsed: 16.1min\n",
      "[Parallel(n_jobs=-1)]: Done 370264 tasks      | elapsed: 16.1min\n",
      "[Parallel(n_jobs=-1)]: Done 371988 tasks      | elapsed: 16.2min\n",
      "[Parallel(n_jobs=-1)]: Done 373720 tasks      | elapsed: 16.3min\n",
      "[Parallel(n_jobs=-1)]: Done 375452 tasks      | elapsed: 16.3min\n",
      "[Parallel(n_jobs=-1)]: Done 377192 tasks      | elapsed: 16.4min\n",
      "[Parallel(n_jobs=-1)]: Done 378932 tasks      | elapsed: 16.5min\n",
      "[Parallel(n_jobs=-1)]: Done 380680 tasks      | elapsed: 16.5min\n",
      "[Parallel(n_jobs=-1)]: Done 382428 tasks      | elapsed: 16.6min\n",
      "[Parallel(n_jobs=-1)]: Done 384184 tasks      | elapsed: 16.7min\n",
      "[Parallel(n_jobs=-1)]: Done 385940 tasks      | elapsed: 16.7min\n",
      "[Parallel(n_jobs=-1)]: Done 387704 tasks      | elapsed: 16.8min\n",
      "[Parallel(n_jobs=-1)]: Done 389468 tasks      | elapsed: 16.9min\n",
      "[Parallel(n_jobs=-1)]: Done 391240 tasks      | elapsed: 17.0min\n",
      "[Parallel(n_jobs=-1)]: Done 393012 tasks      | elapsed: 17.0min\n",
      "[Parallel(n_jobs=-1)]: Done 394792 tasks      | elapsed: 17.1min\n",
      "[Parallel(n_jobs=-1)]: Done 396572 tasks      | elapsed: 17.2min\n",
      "[Parallel(n_jobs=-1)]: Done 398360 tasks      | elapsed: 17.2min\n",
      "[Parallel(n_jobs=-1)]: Done 400148 tasks      | elapsed: 17.3min\n",
      "[Parallel(n_jobs=-1)]: Done 401944 tasks      | elapsed: 17.4min\n",
      "[Parallel(n_jobs=-1)]: Done 403740 tasks      | elapsed: 17.5min\n",
      "[Parallel(n_jobs=-1)]: Done 405544 tasks      | elapsed: 17.5min\n",
      "[Parallel(n_jobs=-1)]: Done 407348 tasks      | elapsed: 17.6min\n",
      "[Parallel(n_jobs=-1)]: Done 409160 tasks      | elapsed: 17.7min\n",
      "[Parallel(n_jobs=-1)]: Done 410972 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done 412792 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done 414612 tasks      | elapsed: 17.9min\n",
      "[Parallel(n_jobs=-1)]: Done 416440 tasks      | elapsed: 18.0min\n",
      "[Parallel(n_jobs=-1)]: Done 418268 tasks      | elapsed: 18.1min\n",
      "[Parallel(n_jobs=-1)]: Done 420104 tasks      | elapsed: 18.1min\n",
      "[Parallel(n_jobs=-1)]: Done 421940 tasks      | elapsed: 18.2min\n",
      "[Parallel(n_jobs=-1)]: Done 423784 tasks      | elapsed: 18.3min\n",
      "[Parallel(n_jobs=-1)]: Done 425628 tasks      | elapsed: 18.4min\n",
      "[Parallel(n_jobs=-1)]: Done 427480 tasks      | elapsed: 18.4min\n",
      "[Parallel(n_jobs=-1)]: Done 429332 tasks      | elapsed: 18.5min\n",
      "[Parallel(n_jobs=-1)]: Done 431192 tasks      | elapsed: 18.6min\n",
      "[Parallel(n_jobs=-1)]: Done 433052 tasks      | elapsed: 18.7min\n",
      "[Parallel(n_jobs=-1)]: Done 434920 tasks      | elapsed: 18.7min\n",
      "[Parallel(n_jobs=-1)]: Done 436788 tasks      | elapsed: 18.8min\n",
      "[Parallel(n_jobs=-1)]: Done 438664 tasks      | elapsed: 18.9min\n",
      "[Parallel(n_jobs=-1)]: Done 440540 tasks      | elapsed: 19.0min\n",
      "[Parallel(n_jobs=-1)]: Done 442424 tasks      | elapsed: 19.1min\n",
      "[Parallel(n_jobs=-1)]: Done 444308 tasks      | elapsed: 19.2min\n",
      "[Parallel(n_jobs=-1)]: Done 446200 tasks      | elapsed: 19.2min\n",
      "[Parallel(n_jobs=-1)]: Done 448092 tasks      | elapsed: 19.3min\n",
      "[Parallel(n_jobs=-1)]: Done 449992 tasks      | elapsed: 19.4min\n",
      "[Parallel(n_jobs=-1)]: Done 451892 tasks      | elapsed: 19.5min\n",
      "[Parallel(n_jobs=-1)]: Done 453800 tasks      | elapsed: 19.6min\n",
      "[Parallel(n_jobs=-1)]: Done 455708 tasks      | elapsed: 19.7min\n",
      "[Parallel(n_jobs=-1)]: Done 457624 tasks      | elapsed: 19.7min\n",
      "[Parallel(n_jobs=-1)]: Done 459540 tasks      | elapsed: 19.8min\n",
      "[Parallel(n_jobs=-1)]: Done 461464 tasks      | elapsed: 19.9min\n",
      "[Parallel(n_jobs=-1)]: Done 463388 tasks      | elapsed: 20.0min\n",
      "[Parallel(n_jobs=-1)]: Done 465320 tasks      | elapsed: 20.1min\n",
      "[Parallel(n_jobs=-1)]: Done 467252 tasks      | elapsed: 20.1min\n",
      "[Parallel(n_jobs=-1)]: Done 469192 tasks      | elapsed: 20.2min\n",
      "[Parallel(n_jobs=-1)]: Done 471132 tasks      | elapsed: 20.3min\n",
      "[Parallel(n_jobs=-1)]: Done 473080 tasks      | elapsed: 20.4min\n",
      "[Parallel(n_jobs=-1)]: Done 475028 tasks      | elapsed: 20.5min\n",
      "[Parallel(n_jobs=-1)]: Done 476984 tasks      | elapsed: 20.5min\n",
      "[Parallel(n_jobs=-1)]: Done 478940 tasks      | elapsed: 20.6min\n",
      "[Parallel(n_jobs=-1)]: Done 480904 tasks      | elapsed: 20.7min\n",
      "[Parallel(n_jobs=-1)]: Done 482868 tasks      | elapsed: 20.8min\n",
      "[Parallel(n_jobs=-1)]: Done 484840 tasks      | elapsed: 20.9min\n",
      "[Parallel(n_jobs=-1)]: Done 486812 tasks      | elapsed: 20.9min\n",
      "[Parallel(n_jobs=-1)]: Done 488792 tasks      | elapsed: 21.0min\n",
      "[Parallel(n_jobs=-1)]: Done 490772 tasks      | elapsed: 21.1min\n",
      "[Parallel(n_jobs=-1)]: Done 492760 tasks      | elapsed: 21.2min\n",
      "[Parallel(n_jobs=-1)]: Done 494748 tasks      | elapsed: 21.3min\n",
      "[Parallel(n_jobs=-1)]: Done 496744 tasks      | elapsed: 21.4min\n",
      "[Parallel(n_jobs=-1)]: Done 498740 tasks      | elapsed: 21.4min\n",
      "[Parallel(n_jobs=-1)]: Done 500744 tasks      | elapsed: 21.5min\n",
      "[Parallel(n_jobs=-1)]: Done 502748 tasks      | elapsed: 21.6min\n",
      "[Parallel(n_jobs=-1)]: Done 504760 tasks      | elapsed: 21.7min\n",
      "[Parallel(n_jobs=-1)]: Done 506772 tasks      | elapsed: 21.8min\n",
      "[Parallel(n_jobs=-1)]: Done 508792 tasks      | elapsed: 21.9min\n",
      "[Parallel(n_jobs=-1)]: Done 510812 tasks      | elapsed: 22.0min\n",
      "[Parallel(n_jobs=-1)]: Done 512840 tasks      | elapsed: 22.0min\n",
      "[Parallel(n_jobs=-1)]: Done 514868 tasks      | elapsed: 22.1min\n",
      "[Parallel(n_jobs=-1)]: Done 516904 tasks      | elapsed: 22.2min\n",
      "[Parallel(n_jobs=-1)]: Done 518940 tasks      | elapsed: 22.3min\n",
      "[Parallel(n_jobs=-1)]: Done 520984 tasks      | elapsed: 22.4min\n",
      "[Parallel(n_jobs=-1)]: Done 523028 tasks      | elapsed: 22.5min\n",
      "[Parallel(n_jobs=-1)]: Done 525080 tasks      | elapsed: 22.6min\n",
      "[Parallel(n_jobs=-1)]: Done 527132 tasks      | elapsed: 22.6min\n",
      "[Parallel(n_jobs=-1)]: Done 529192 tasks      | elapsed: 22.7min\n",
      "[Parallel(n_jobs=-1)]: Done 531252 tasks      | elapsed: 22.8min\n",
      "[Parallel(n_jobs=-1)]: Done 533320 tasks      | elapsed: 22.9min\n",
      "[Parallel(n_jobs=-1)]: Done 535388 tasks      | elapsed: 23.0min\n",
      "[Parallel(n_jobs=-1)]: Done 537464 tasks      | elapsed: 23.1min\n",
      "[Parallel(n_jobs=-1)]: Done 539540 tasks      | elapsed: 23.2min\n",
      "[Parallel(n_jobs=-1)]: Done 541624 tasks      | elapsed: 23.3min\n",
      "[Parallel(n_jobs=-1)]: Done 543708 tasks      | elapsed: 23.4min\n",
      "[Parallel(n_jobs=-1)]: Done 545800 tasks      | elapsed: 23.4min\n",
      "[Parallel(n_jobs=-1)]: Done 547892 tasks      | elapsed: 23.5min\n",
      "[Parallel(n_jobs=-1)]: Done 549992 tasks      | elapsed: 23.6min\n",
      "[Parallel(n_jobs=-1)]: Done 552092 tasks      | elapsed: 23.7min\n",
      "[Parallel(n_jobs=-1)]: Done 554200 tasks      | elapsed: 23.8min\n",
      "[Parallel(n_jobs=-1)]: Done 556308 tasks      | elapsed: 23.9min\n",
      "[Parallel(n_jobs=-1)]: Done 558424 tasks      | elapsed: 24.0min\n",
      "[Parallel(n_jobs=-1)]: Done 560540 tasks      | elapsed: 24.0min\n",
      "[Parallel(n_jobs=-1)]: Done 562664 tasks      | elapsed: 24.1min\n",
      "[Parallel(n_jobs=-1)]: Done 564788 tasks      | elapsed: 24.2min\n",
      "[Parallel(n_jobs=-1)]: Done 566920 tasks      | elapsed: 24.3min\n",
      "[Parallel(n_jobs=-1)]: Done 569052 tasks      | elapsed: 24.4min\n",
      "[Parallel(n_jobs=-1)]: Done 571192 tasks      | elapsed: 24.5min\n",
      "[Parallel(n_jobs=-1)]: Done 573332 tasks      | elapsed: 24.6min\n",
      "[Parallel(n_jobs=-1)]: Done 575480 tasks      | elapsed: 24.6min\n",
      "[Parallel(n_jobs=-1)]: Done 577628 tasks      | elapsed: 24.7min\n",
      "[Parallel(n_jobs=-1)]: Done 579784 tasks      | elapsed: 24.8min\n",
      "[Parallel(n_jobs=-1)]: Done 581940 tasks      | elapsed: 24.9min\n",
      "[Parallel(n_jobs=-1)]: Done 584104 tasks      | elapsed: 25.0min\n",
      "[Parallel(n_jobs=-1)]: Done 586268 tasks      | elapsed: 25.1min\n",
      "[Parallel(n_jobs=-1)]: Done 588440 tasks      | elapsed: 25.2min\n",
      "[Parallel(n_jobs=-1)]: Done 590612 tasks      | elapsed: 25.3min\n",
      "[Parallel(n_jobs=-1)]: Done 592792 tasks      | elapsed: 25.4min\n",
      "[Parallel(n_jobs=-1)]: Done 594972 tasks      | elapsed: 25.4min\n",
      "[Parallel(n_jobs=-1)]: Done 597160 tasks      | elapsed: 25.5min\n",
      "[Parallel(n_jobs=-1)]: Done 599348 tasks      | elapsed: 25.6min\n",
      "[Parallel(n_jobs=-1)]: Done 601544 tasks      | elapsed: 25.7min\n",
      "[Parallel(n_jobs=-1)]: Done 603740 tasks      | elapsed: 25.8min\n",
      "[Parallel(n_jobs=-1)]: Done 605944 tasks      | elapsed: 25.9min\n",
      "[Parallel(n_jobs=-1)]: Done 608148 tasks      | elapsed: 26.0min\n",
      "[Parallel(n_jobs=-1)]: Done 610360 tasks      | elapsed: 26.1min\n",
      "[Parallel(n_jobs=-1)]: Done 612572 tasks      | elapsed: 26.2min\n",
      "[Parallel(n_jobs=-1)]: Done 614792 tasks      | elapsed: 26.3min\n",
      "[Parallel(n_jobs=-1)]: Done 617012 tasks      | elapsed: 26.3min\n",
      "[Parallel(n_jobs=-1)]: Done 619240 tasks      | elapsed: 26.4min\n",
      "[Parallel(n_jobs=-1)]: Done 621468 tasks      | elapsed: 26.5min\n",
      "[Parallel(n_jobs=-1)]: Done 623704 tasks      | elapsed: 26.6min\n",
      "[Parallel(n_jobs=-1)]: Done 625940 tasks      | elapsed: 26.7min\n",
      "[Parallel(n_jobs=-1)]: Done 628184 tasks      | elapsed: 26.8min\n",
      "[Parallel(n_jobs=-1)]: Done 630428 tasks      | elapsed: 26.9min\n",
      "[Parallel(n_jobs=-1)]: Done 632680 tasks      | elapsed: 27.0min\n",
      "[Parallel(n_jobs=-1)]: Done 634932 tasks      | elapsed: 27.1min\n",
      "[Parallel(n_jobs=-1)]: Done 637192 tasks      | elapsed: 27.2min\n",
      "[Parallel(n_jobs=-1)]: Done 639452 tasks      | elapsed: 27.2min\n",
      "[Parallel(n_jobs=-1)]: Done 641720 tasks      | elapsed: 27.3min\n",
      "[Parallel(n_jobs=-1)]: Done 643988 tasks      | elapsed: 27.4min\n",
      "[Parallel(n_jobs=-1)]: Done 646264 tasks      | elapsed: 27.5min\n",
      "[Parallel(n_jobs=-1)]: Done 648540 tasks      | elapsed: 27.6min\n",
      "[Parallel(n_jobs=-1)]: Done 650824 tasks      | elapsed: 27.7min\n",
      "[Parallel(n_jobs=-1)]: Done 653108 tasks      | elapsed: 27.8min\n",
      "[Parallel(n_jobs=-1)]: Done 655400 tasks      | elapsed: 27.8min\n",
      "[Parallel(n_jobs=-1)]: Done 657692 tasks      | elapsed: 27.9min\n",
      "[Parallel(n_jobs=-1)]: Done 659992 tasks      | elapsed: 28.0min\n",
      "[Parallel(n_jobs=-1)]: Done 662292 tasks      | elapsed: 28.1min\n",
      "[Parallel(n_jobs=-1)]: Done 664600 tasks      | elapsed: 28.2min\n",
      "[Parallel(n_jobs=-1)]: Done 666908 tasks      | elapsed: 28.3min\n",
      "[Parallel(n_jobs=-1)]: Done 669224 tasks      | elapsed: 28.4min\n",
      "[Parallel(n_jobs=-1)]: Done 671540 tasks      | elapsed: 28.5min\n",
      "[Parallel(n_jobs=-1)]: Done 673864 tasks      | elapsed: 28.6min\n",
      "[Parallel(n_jobs=-1)]: Done 676188 tasks      | elapsed: 28.7min\n",
      "[Parallel(n_jobs=-1)]: Done 678520 tasks      | elapsed: 28.8min\n",
      "[Parallel(n_jobs=-1)]: Done 680852 tasks      | elapsed: 28.8min\n",
      "[Parallel(n_jobs=-1)]: Done 683192 tasks      | elapsed: 28.9min\n",
      "[Parallel(n_jobs=-1)]: Done 685532 tasks      | elapsed: 29.0min\n",
      "[Parallel(n_jobs=-1)]: Done 687880 tasks      | elapsed: 29.1min\n",
      "[Parallel(n_jobs=-1)]: Done 690228 tasks      | elapsed: 29.2min\n",
      "[Parallel(n_jobs=-1)]: Done 692584 tasks      | elapsed: 29.3min\n",
      "[Parallel(n_jobs=-1)]: Done 694940 tasks      | elapsed: 29.4min\n",
      "[Parallel(n_jobs=-1)]: Done 697304 tasks      | elapsed: 29.5min\n",
      "[Parallel(n_jobs=-1)]: Done 699668 tasks      | elapsed: 29.6min\n",
      "[Parallel(n_jobs=-1)]: Done 702040 tasks      | elapsed: 29.6min\n",
      "[Parallel(n_jobs=-1)]: Done 704412 tasks      | elapsed: 29.7min\n",
      "[Parallel(n_jobs=-1)]: Done 706792 tasks      | elapsed: 29.8min\n",
      "[Parallel(n_jobs=-1)]: Done 709172 tasks      | elapsed: 29.9min\n",
      "[Parallel(n_jobs=-1)]: Done 711560 tasks      | elapsed: 30.0min\n",
      "[Parallel(n_jobs=-1)]: Done 713948 tasks      | elapsed: 30.1min\n",
      "[Parallel(n_jobs=-1)]: Done 716344 tasks      | elapsed: 30.2min\n",
      "[Parallel(n_jobs=-1)]: Done 718740 tasks      | elapsed: 30.3min\n",
      "[Parallel(n_jobs=-1)]: Done 721144 tasks      | elapsed: 30.4min\n",
      "[Parallel(n_jobs=-1)]: Done 723548 tasks      | elapsed: 30.5min\n",
      "[Parallel(n_jobs=-1)]: Done 725960 tasks      | elapsed: 30.6min\n",
      "[Parallel(n_jobs=-1)]: Done 728372 tasks      | elapsed: 30.6min\n",
      "[Parallel(n_jobs=-1)]: Done 730792 tasks      | elapsed: 30.7min\n",
      "[Parallel(n_jobs=-1)]: Done 733212 tasks      | elapsed: 30.8min\n",
      "[Parallel(n_jobs=-1)]: Done 735640 tasks      | elapsed: 30.9min\n",
      "[Parallel(n_jobs=-1)]: Done 738068 tasks      | elapsed: 31.0min\n",
      "[Parallel(n_jobs=-1)]: Done 740504 tasks      | elapsed: 31.1min\n",
      "[Parallel(n_jobs=-1)]: Done 742940 tasks      | elapsed: 31.2min\n",
      "[Parallel(n_jobs=-1)]: Done 745384 tasks      | elapsed: 31.3min\n",
      "[Parallel(n_jobs=-1)]: Done 747828 tasks      | elapsed: 31.4min\n",
      "[Parallel(n_jobs=-1)]: Done 750280 tasks      | elapsed: 31.4min\n",
      "[Parallel(n_jobs=-1)]: Done 752732 tasks      | elapsed: 31.5min\n",
      "[Parallel(n_jobs=-1)]: Done 755192 tasks      | elapsed: 31.6min\n",
      "[Parallel(n_jobs=-1)]: Done 757652 tasks      | elapsed: 31.7min\n",
      "[Parallel(n_jobs=-1)]: Done 760120 tasks      | elapsed: 31.8min\n",
      "[Parallel(n_jobs=-1)]: Done 762588 tasks      | elapsed: 31.9min\n",
      "[Parallel(n_jobs=-1)]: Done 765064 tasks      | elapsed: 32.0min\n",
      "[Parallel(n_jobs=-1)]: Done 767540 tasks      | elapsed: 32.1min\n",
      "[Parallel(n_jobs=-1)]: Done 770024 tasks      | elapsed: 32.2min\n",
      "[Parallel(n_jobs=-1)]: Done 772508 tasks      | elapsed: 32.3min\n",
      "[Parallel(n_jobs=-1)]: Done 775000 tasks      | elapsed: 32.4min\n",
      "[Parallel(n_jobs=-1)]: Done 777492 tasks      | elapsed: 32.5min\n",
      "[Parallel(n_jobs=-1)]: Done 779992 tasks      | elapsed: 32.6min\n",
      "[Parallel(n_jobs=-1)]: Done 782492 tasks      | elapsed: 32.6min\n",
      "[Parallel(n_jobs=-1)]: Done 785000 tasks      | elapsed: 32.7min\n",
      "[Parallel(n_jobs=-1)]: Done 787508 tasks      | elapsed: 32.8min\n",
      "[Parallel(n_jobs=-1)]: Done 790024 tasks      | elapsed: 32.9min\n",
      "[Parallel(n_jobs=-1)]: Done 792540 tasks      | elapsed: 33.0min\n",
      "[Parallel(n_jobs=-1)]: Done 795064 tasks      | elapsed: 33.1min\n",
      "[Parallel(n_jobs=-1)]: Done 797588 tasks      | elapsed: 33.2min\n",
      "[Parallel(n_jobs=-1)]: Done 800120 tasks      | elapsed: 33.3min\n",
      "[Parallel(n_jobs=-1)]: Done 802652 tasks      | elapsed: 33.4min\n",
      "[Parallel(n_jobs=-1)]: Done 805192 tasks      | elapsed: 33.5min\n",
      "[Parallel(n_jobs=-1)]: Done 807732 tasks      | elapsed: 33.6min\n",
      "[Parallel(n_jobs=-1)]: Done 810280 tasks      | elapsed: 33.7min\n",
      "[Parallel(n_jobs=-1)]: Done 812828 tasks      | elapsed: 33.8min\n",
      "[Parallel(n_jobs=-1)]: Done 815384 tasks      | elapsed: 33.9min\n",
      "[Parallel(n_jobs=-1)]: Done 817940 tasks      | elapsed: 34.0min\n",
      "[Parallel(n_jobs=-1)]: Done 820504 tasks      | elapsed: 34.1min\n",
      "[Parallel(n_jobs=-1)]: Done 823068 tasks      | elapsed: 34.2min\n",
      "[Parallel(n_jobs=-1)]: Done 825640 tasks      | elapsed: 34.3min\n",
      "[Parallel(n_jobs=-1)]: Done 828212 tasks      | elapsed: 34.4min\n",
      "[Parallel(n_jobs=-1)]: Done 830792 tasks      | elapsed: 34.5min\n",
      "[Parallel(n_jobs=-1)]: Done 833372 tasks      | elapsed: 34.6min\n",
      "[Parallel(n_jobs=-1)]: Done 835960 tasks      | elapsed: 34.7min\n",
      "[Parallel(n_jobs=-1)]: Done 838548 tasks      | elapsed: 34.8min\n",
      "[Parallel(n_jobs=-1)]: Done 841144 tasks      | elapsed: 34.9min\n",
      "[Parallel(n_jobs=-1)]: Done 843740 tasks      | elapsed: 35.0min\n",
      "[Parallel(n_jobs=-1)]: Done 846344 tasks      | elapsed: 35.1min\n",
      "[Parallel(n_jobs=-1)]: Done 848948 tasks      | elapsed: 35.2min\n",
      "[Parallel(n_jobs=-1)]: Done 851560 tasks      | elapsed: 35.3min\n",
      "[Parallel(n_jobs=-1)]: Done 854172 tasks      | elapsed: 35.4min\n",
      "[Parallel(n_jobs=-1)]: Done 856792 tasks      | elapsed: 35.5min\n",
      "[Parallel(n_jobs=-1)]: Done 859412 tasks      | elapsed: 35.6min\n",
      "[Parallel(n_jobs=-1)]: Done 862040 tasks      | elapsed: 35.7min\n",
      "[Parallel(n_jobs=-1)]: Done 864668 tasks      | elapsed: 35.8min\n",
      "[Parallel(n_jobs=-1)]: Done 867304 tasks      | elapsed: 35.8min\n",
      "[Parallel(n_jobs=-1)]: Done 869940 tasks      | elapsed: 35.9min\n",
      "[Parallel(n_jobs=-1)]: Done 872584 tasks      | elapsed: 36.0min\n",
      "[Parallel(n_jobs=-1)]: Done 875228 tasks      | elapsed: 36.1min\n",
      "[Parallel(n_jobs=-1)]: Done 877880 tasks      | elapsed: 36.2min\n",
      "[Parallel(n_jobs=-1)]: Done 880532 tasks      | elapsed: 36.3min\n",
      "[Parallel(n_jobs=-1)]: Done 883192 tasks      | elapsed: 36.4min\n",
      "[Parallel(n_jobs=-1)]: Done 885852 tasks      | elapsed: 36.5min\n",
      "[Parallel(n_jobs=-1)]: Done 888520 tasks      | elapsed: 36.6min\n",
      "[Parallel(n_jobs=-1)]: Done 891188 tasks      | elapsed: 36.7min\n",
      "[Parallel(n_jobs=-1)]: Done 893864 tasks      | elapsed: 36.8min\n",
      "[Parallel(n_jobs=-1)]: Done 896540 tasks      | elapsed: 36.9min\n",
      "[Parallel(n_jobs=-1)]: Done 899224 tasks      | elapsed: 37.0min\n",
      "[Parallel(n_jobs=-1)]: Done 901908 tasks      | elapsed: 37.1min\n",
      "[Parallel(n_jobs=-1)]: Done 904600 tasks      | elapsed: 37.2min\n",
      "[Parallel(n_jobs=-1)]: Done 907292 tasks      | elapsed: 37.3min\n",
      "[Parallel(n_jobs=-1)]: Done 909992 tasks      | elapsed: 37.4min\n",
      "[Parallel(n_jobs=-1)]: Done 912692 tasks      | elapsed: 37.5min\n",
      "[Parallel(n_jobs=-1)]: Done 915400 tasks      | elapsed: 37.6min\n",
      "[Parallel(n_jobs=-1)]: Done 918108 tasks      | elapsed: 37.7min\n",
      "[Parallel(n_jobs=-1)]: Done 920824 tasks      | elapsed: 37.8min\n",
      "[Parallel(n_jobs=-1)]: Done 923540 tasks      | elapsed: 37.9min\n",
      "[Parallel(n_jobs=-1)]: Done 926264 tasks      | elapsed: 38.0min\n",
      "[Parallel(n_jobs=-1)]: Done 928988 tasks      | elapsed: 38.1min\n",
      "[Parallel(n_jobs=-1)]: Done 931720 tasks      | elapsed: 38.2min\n",
      "[Parallel(n_jobs=-1)]: Done 934452 tasks      | elapsed: 38.3min\n",
      "[Parallel(n_jobs=-1)]: Done 937192 tasks      | elapsed: 38.4min\n",
      "[Parallel(n_jobs=-1)]: Done 939932 tasks      | elapsed: 38.5min\n",
      "[Parallel(n_jobs=-1)]: Done 942680 tasks      | elapsed: 38.6min\n",
      "[Parallel(n_jobs=-1)]: Done 945428 tasks      | elapsed: 38.7min\n",
      "[Parallel(n_jobs=-1)]: Done 948184 tasks      | elapsed: 38.8min\n",
      "[Parallel(n_jobs=-1)]: Done 950940 tasks      | elapsed: 38.9min\n",
      "[Parallel(n_jobs=-1)]: Done 953704 tasks      | elapsed: 39.0min\n",
      "[Parallel(n_jobs=-1)]: Done 956468 tasks      | elapsed: 39.1min\n",
      "[Parallel(n_jobs=-1)]: Done 959240 tasks      | elapsed: 39.2min\n",
      "[Parallel(n_jobs=-1)]: Done 962012 tasks      | elapsed: 39.3min\n",
      "[Parallel(n_jobs=-1)]: Done 964792 tasks      | elapsed: 39.4min\n",
      "[Parallel(n_jobs=-1)]: Done 967572 tasks      | elapsed: 39.5min\n",
      "[Parallel(n_jobs=-1)]: Done 970360 tasks      | elapsed: 39.6min\n",
      "[Parallel(n_jobs=-1)]: Done 973148 tasks      | elapsed: 39.7min\n",
      "[Parallel(n_jobs=-1)]: Done 975944 tasks      | elapsed: 39.8min\n",
      "[Parallel(n_jobs=-1)]: Done 978740 tasks      | elapsed: 39.9min\n",
      "[Parallel(n_jobs=-1)]: Done 981544 tasks      | elapsed: 40.0min\n",
      "[Parallel(n_jobs=-1)]: Done 984348 tasks      | elapsed: 40.1min\n",
      "[Parallel(n_jobs=-1)]: Done 987160 tasks      | elapsed: 40.2min\n",
      "[Parallel(n_jobs=-1)]: Done 989972 tasks      | elapsed: 40.3min\n",
      "[Parallel(n_jobs=-1)]: Done 992792 tasks      | elapsed: 40.4min\n",
      "[Parallel(n_jobs=-1)]: Done 995612 tasks      | elapsed: 40.5min\n",
      "[Parallel(n_jobs=-1)]: Done 998440 tasks      | elapsed: 40.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1001268 tasks      | elapsed: 40.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1004104 tasks      | elapsed: 40.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1006940 tasks      | elapsed: 40.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1009784 tasks      | elapsed: 41.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1012628 tasks      | elapsed: 41.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1015480 tasks      | elapsed: 41.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1018332 tasks      | elapsed: 41.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1021192 tasks      | elapsed: 41.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1024052 tasks      | elapsed: 41.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1026920 tasks      | elapsed: 41.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1029788 tasks      | elapsed: 41.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1032664 tasks      | elapsed: 41.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1035540 tasks      | elapsed: 42.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1038424 tasks      | elapsed: 42.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1041308 tasks      | elapsed: 42.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1044200 tasks      | elapsed: 42.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1047092 tasks      | elapsed: 42.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1049992 tasks      | elapsed: 42.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1052892 tasks      | elapsed: 42.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1055800 tasks      | elapsed: 42.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1058708 tasks      | elapsed: 42.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1061624 tasks      | elapsed: 42.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1064540 tasks      | elapsed: 43.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1067464 tasks      | elapsed: 43.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1070388 tasks      | elapsed: 43.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1073320 tasks      | elapsed: 43.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1076252 tasks      | elapsed: 43.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1079192 tasks      | elapsed: 43.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1082132 tasks      | elapsed: 43.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1085080 tasks      | elapsed: 43.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1088028 tasks      | elapsed: 43.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1090984 tasks      | elapsed: 44.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1093940 tasks      | elapsed: 44.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1096904 tasks      | elapsed: 44.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1099868 tasks      | elapsed: 44.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1102840 tasks      | elapsed: 44.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1105812 tasks      | elapsed: 44.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1108792 tasks      | elapsed: 44.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1111772 tasks      | elapsed: 44.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1114760 tasks      | elapsed: 44.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1117748 tasks      | elapsed: 44.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1120744 tasks      | elapsed: 45.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1123740 tasks      | elapsed: 45.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1126744 tasks      | elapsed: 45.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1129748 tasks      | elapsed: 45.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1132760 tasks      | elapsed: 45.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1135772 tasks      | elapsed: 45.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1138792 tasks      | elapsed: 45.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1141812 tasks      | elapsed: 45.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1144840 tasks      | elapsed: 45.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1147868 tasks      | elapsed: 46.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1150904 tasks      | elapsed: 46.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1153940 tasks      | elapsed: 46.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1156984 tasks      | elapsed: 46.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1160028 tasks      | elapsed: 46.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1163080 tasks      | elapsed: 46.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1166132 tasks      | elapsed: 46.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1169192 tasks      | elapsed: 46.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1172252 tasks      | elapsed: 46.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1175320 tasks      | elapsed: 47.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1178388 tasks      | elapsed: 47.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1181464 tasks      | elapsed: 47.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1184540 tasks      | elapsed: 47.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1187624 tasks      | elapsed: 47.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1190708 tasks      | elapsed: 47.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1193800 tasks      | elapsed: 47.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1196892 tasks      | elapsed: 47.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1199992 tasks      | elapsed: 48.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1203092 tasks      | elapsed: 48.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1206200 tasks      | elapsed: 48.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1209308 tasks      | elapsed: 48.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1212424 tasks      | elapsed: 48.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1215540 tasks      | elapsed: 48.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1218664 tasks      | elapsed: 48.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1221788 tasks      | elapsed: 48.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1224920 tasks      | elapsed: 48.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1228052 tasks      | elapsed: 49.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1231192 tasks      | elapsed: 49.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1234332 tasks      | elapsed: 49.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1237480 tasks      | elapsed: 49.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1240628 tasks      | elapsed: 49.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1243784 tasks      | elapsed: 49.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1246940 tasks      | elapsed: 49.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1250104 tasks      | elapsed: 49.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1253268 tasks      | elapsed: 50.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1256440 tasks      | elapsed: 50.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1259612 tasks      | elapsed: 50.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1262792 tasks      | elapsed: 50.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1265972 tasks      | elapsed: 50.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1269160 tasks      | elapsed: 50.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1272348 tasks      | elapsed: 50.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1275544 tasks      | elapsed: 50.8min\n",
      "[Parallel(n_jobs=-1)]: Done 1278740 tasks      | elapsed: 50.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1281944 tasks      | elapsed: 51.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1285148 tasks      | elapsed: 51.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1288360 tasks      | elapsed: 51.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1291572 tasks      | elapsed: 51.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1294792 tasks      | elapsed: 51.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1298012 tasks      | elapsed: 51.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1301240 tasks      | elapsed: 51.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1304468 tasks      | elapsed: 51.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1307704 tasks      | elapsed: 52.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1310940 tasks      | elapsed: 52.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1314184 tasks      | elapsed: 52.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1317428 tasks      | elapsed: 52.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1320680 tasks      | elapsed: 52.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1323932 tasks      | elapsed: 52.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1327192 tasks      | elapsed: 52.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1330452 tasks      | elapsed: 52.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1333720 tasks      | elapsed: 53.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1336988 tasks      | elapsed: 53.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1340264 tasks      | elapsed: 53.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1343540 tasks      | elapsed: 53.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1346824 tasks      | elapsed: 53.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1350108 tasks      | elapsed: 53.6min\n",
      "[Parallel(n_jobs=-1)]: Done 1353400 tasks      | elapsed: 53.7min\n",
      "[Parallel(n_jobs=-1)]: Done 1356692 tasks      | elapsed: 53.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1359992 tasks      | elapsed: 54.0min\n",
      "[Parallel(n_jobs=-1)]: Done 1363292 tasks      | elapsed: 54.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1366600 tasks      | elapsed: 54.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1369908 tasks      | elapsed: 54.4min\n",
      "[Parallel(n_jobs=-1)]: Done 1370833 out of 1370872 | elapsed: 54.4min remaining:    0.0s\n",
      "[Parallel(n_jobs=-1)]: Done 1370872 out of 1370872 | elapsed: 54.4min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1370872"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def filter_img(x: str) -> Union[str, None]:\n",
    "    try:\n",
    "        process_image_2_pixel_value(x)\n",
    "        # value = read_image(x, mode=ImageReadMode.RGB)\n",
    "        # if len(value.shape) ==3:\n",
    "        return x\n",
    "\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        return None\n",
    "\n",
    "# read_image(all_image_path_list[0], mode=ImageReadMode.RGB).shape\n",
    "# clean_image_path_list = [filter_img(i) for i in tqdm(all_image_path_list[:100])]\n",
    "clean_image_path_list = Parallel(n_jobs=-1, verbose=10)(delayed(filter_img)(i) for i in all_image_path_list)\n",
    "len(clean_image_path_list)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1309540"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_image_path_list = [i for i in clean_image_path_list if i is not None]\n",
    "len(clean_image_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(\"bigdata/temp_dir\", exist_ok=True)\n",
    "pd.DataFrame({'image_path':clean_image_path_list}).to_csv(\"bigdata/temp_dir/temp_image_url.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mynet2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "110bc624a448454d574a0cd6cc76359fd86f75739e493913b3d71c2e04f2ffdb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
